{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# 一个完整的机器学习项目\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## 对数据下载并简单处理\n",
    "\n",
    "推荐一篇博客关于这个内容的详细介绍 博客https://blog.csdn.net/weixin_30687587/article/details/102403315\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:00.619379Z",
     "start_time": "2023-02-07T03:44:00.617632Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# import os\n",
    "# import tarfile  # 解压文件用\n",
    "# import urllib\n",
    "\n",
    "# DOWNLOAD_ROOT = \"https://raw.githubusercontent.com/ageron/handson-ml2/master/\"\n",
    "# HOUSING_PATH = os.path.join(\"datasets\", \"housing\")\n",
    "# HOUSING_URL = DOWNLOAD_ROOT+\"datasets/housing/housing.tgz\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### 读取数据\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:00.834050Z",
     "start_time": "2023-02-07T03:44:00.620857Z"
    },
    "pycharm": {
     "name": "#%%\n"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "data = pd.read_csv(\"housing/housing.csv\")\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:00.841587Z",
     "start_time": "2023-02-07T03:44:00.835321Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "data.info()   # 展示数据的详细信息"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:00.846646Z",
     "start_time": "2023-02-07T03:44:00.842596Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "data[\"ocean_proximity\"].value_counts()  # 对ocean_proximity进行统计"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:00.867221Z",
     "start_time": "2023-02-07T03:44:00.848797Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# 查看属性值\n",
    "data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:01.712134Z",
     "start_time": "2023-02-07T03:44:00.868157Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# 用图表示数据(用频数分布直方图表示)\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "data.hist(bins=50, figsize=(20, 15))   # bins指的是直方图的柱的个数,figsize指的是图片的长与宽的长度\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### 创建测试集\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:01.715131Z",
     "start_time": "2023-02-07T03:44:01.713100Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "# 自定义划分函数\n",
    "\n",
    "\n",
    "def split_train_test(data, test_rate):\n",
    "    # 设置随机种子(保证每次生成的随机序列都保持一致)  如果不加随机种子,会导致每次重新运行时产生不同的随机序列\n",
    "    np.random.seed(42)\n",
    "    shuffled_indices = np.random.permutation(len(data))   # 产生随机序列\n",
    "    test_set_size = int(len(data)*test_rate)   # 测试集的个数\n",
    "    test_indices = shuffled_indices[:test_set_size]  # 测试集的数据\n",
    "    train_indices = shuffled_indices[test_set_size:]  # 训练集的数据\n",
    "    # print(train_indices, test_indices)\n",
    "    return data.iloc[train_indices], data.iloc[test_indices]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:01.718438Z",
     "start_time": "2023-02-07T03:44:01.715786Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "train_set, test_set = split_train_test(data, 0.2)    # 生成测试集与训练集 4:1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:01.727353Z",
     "start_time": "2023-02-07T03:44:01.719320Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "train_set   # 训练集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:01.736905Z",
     "start_time": "2023-02-07T03:44:01.728713Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "test_set   # 测试集"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### 针对更新数据进行创建测试集\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:01.741949Z",
     "start_time": "2023-02-07T03:44:01.739714Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "import hashlib\n",
    "\n",
    "# 检测标识符判断是否进入测试集\n",
    "\n",
    "\n",
    "def test_set_check(indentifier, test_rate, hash):\n",
    "    return hash(np.int64(indentifier)).digest()[-1] < 256 * test_rate\n",
    "\n",
    "# id_column是索引的那一列属性,这里以\"index\"这一列进行验证\n",
    "\n",
    "\n",
    "def split_train_test_by_id(data, test_rate, id_column, hash=hashlib.md5):\n",
    "    ids = data[id_column]\n",
    "    in_test_set = ids.apply(lambda id_: test_set_check(id_, test_rate, hash))\n",
    "    # print(in_test_set)  # 得到是布尔类型的值\n",
    "    return data.loc[~in_test_set], data.loc[in_test_set]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:01.751383Z",
     "start_time": "2023-02-07T03:44:01.743031Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:01.761519Z",
     "start_time": "2023-02-07T03:44:01.752805Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "data.reset_index()    # 增加一个index列"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:01.784691Z",
     "start_time": "2023-02-07T03:44:01.762509Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "data_with_id = data.reset_index()   # 让index作为ID\n",
    "train_set, test_set = split_train_test_by_id(\n",
    "    data_with_id, 0.2, \"index\")   # 让index作为标识符"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:01.794132Z",
     "start_time": "2023-02-07T03:44:01.785677Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "train_set   # 训练集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:01.803478Z",
     "start_time": "2023-02-07T03:44:01.795312Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "test_set  # 测试集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:03.494428Z",
     "start_time": "2023-02-07T03:44:01.804609Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# 由于以上的方法会导致当新加数据时必须从末尾添加,导致不方便,因此我们可以使用时sklearn也提供了生成test_set的方法\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "train_set, test_set = train_test_split(\n",
    "    data, test_size=0.2, random_state=42)  # 第二个参数时比例,第三个参数时随机种子"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:03.510072Z",
     "start_time": "2023-02-07T03:44:03.495369Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "train_set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:03.525162Z",
     "start_time": "2023-02-07T03:44:03.511878Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "test_set"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### 采用分层抽样法\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:03.652039Z",
     "start_time": "2023-02-07T03:44:03.526367Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# 创建新的标签来做直方图   labels是分割之后的标签  相当于把数据分好了类\n",
    "data[\"income_cat\"] = pd.cut(data[\"median_income\"], bins=[\n",
    "                            0, 1.5, 3.0, 4.5, 6, np.inf], labels=[1, 2, 3, 4, 5])  # 进行数据的处理,相当于高中的分割数据区间\n",
    "data[\"income_cat\"].hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:03.661581Z",
     "start_time": "2023-02-07T03:44:03.652941Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# 根据已分好类的income_cat来进行分类抽样\n",
    "from sklearn.model_selection import StratifiedShuffleSplit\n",
    "# 第一个参数是将训练数据分成train/test对的组数，可根据需要进行设置，默认为10将训练数据分成train/test对的组数，可根据需要进行设置，默认为10\n",
    "split = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42)\n",
    "for train_index, test_index in split.split(data, data[\"income_cat\"]):\n",
    "    strat_train_set = data.loc[train_index]\n",
    "    strat_test_set = data.loc[test_index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:03.666390Z",
     "start_time": "2023-02-07T03:44:03.662625Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "strat_train_set[\"income_cat\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:03.670807Z",
     "start_time": "2023-02-07T03:44:03.667378Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "strat_test_set[\"income_cat\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:03.675714Z",
     "start_time": "2023-02-07T03:44:03.672046Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# 看看测试集中收入类的比例,每个分类的比例\n",
    "strat_test_set[\"income_cat\"].value_counts() / len(strat_test_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:03.679532Z",
     "start_time": "2023-02-07T03:44:03.676696Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# 删除income_cat属性\n",
    "for s in (strat_train_set, strat_test_set):\n",
    "    s.drop(\"income_cat\", axis=1, inplace=True)\n",
    "  # 在俩个测试集与训练集的列的方向删除income_cat的属性, 如果将inplace值设定为True,则原数组内容直接被改变。如果将inplace值设定为True,则原数组内容直接被改变。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:03.682698Z",
     "start_time": "2023-02-07T03:44:03.680631Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "housing = strat_train_set.copy()   # 训练数据集"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### 数据的可视化\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:03.692277Z",
     "start_time": "2023-02-07T03:44:03.683708Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "housing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:03.826393Z",
     "start_time": "2023-02-07T03:44:03.693183Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# kind=\"scatter\"这个指的是画散点图\n",
    "# kind : str\n",
    "#     The kind of plot to produce:\n",
    "\n",
    "#     - 'line' : line plot (default)     折线图\n",
    "#     - 'bar' : vertical bar plot        条形图\n",
    "#     - 'barh' : horizontal bar plot      横向条形图\n",
    "#     - 'hist' : histogram                柱状图\n",
    "#     - 'box' : boxplot              箱线图\n",
    "#     - 'kde' : Kernel Density Estimation plot           # Kernel 的密度估计图，主要对柱状图添加Kernel 概率密度线\n",
    "#     - 'density' : same as 'kde'\n",
    "#     - 'area' : area plot           面积图\n",
    "#     - 'pie' : pie plot          饼图\n",
    "#     - 'scatter' : scatter plot    散点图\n",
    "#     - 'hexbin' : hexbin plot.\n",
    "housing.plot(kind=\"scatter\", x=\"longitude\", y=\"latitude\", figsize=(\n",
    "    20, 12))  # 以散点图的形式来画,x,y为longitude,latitudelatitude"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:03.968681Z",
     "start_time": "2023-02-07T03:44:03.831896Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "housing.plot(kind=\"scatter\", x=\"longitude\", y=\"latitude\", figsize=(\n",
    "    20, 12), alpha=0.1)   # alpha=0.1是为了更好看清高密度的数据,就是调整点的透明度"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:04.399562Z",
     "start_time": "2023-02-07T03:44:03.969553Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# 在散点图中\n",
    "# s: 指定散点图点的大小，默认为20，通过新传入的变量，实现气泡图的绘制（size的首字母） 人口规模\n",
    "# c: 指定散点图点的颜色，默认为蓝色（color的首字母）\n",
    "# marker: 指定散点图点的形状，默认为圆形\n",
    "# cmap: 指定色图，只有当c参数是一个浮点型的数组时才起作用        jet颜色图\n",
    "# norm: 指定数据亮度， 标准化到0~1之间，使用该参数仍需要c为浮点型的数组\n",
    "# vmin、vmax: 亮度设置，与norm类似，如果使用了norm则该参数无效\n",
    "# alpha: 设置散点的透明度\n",
    "# edgecolors: 设置散点边界线的颜色\n",
    "# linewidths: 设置散点边界线的粗细\n",
    "# colorbar:当我们给图配渐变色时，常常需要在图旁边把colorbar显示出来。\n",
    "# plt.legend（）函数主要的作用就是给图加上图例，\n",
    "housing.plot(kind=\"scatter\", x=\"longitude\", y=\"latitude\", alpha=0.4,\n",
    "             s=housing[\"population\"]/100, label=\"population\", figsize=(20, 12),\n",
    "             c=\"median_house_value\", cmap=plt.get_cmap(\"jet\"), colorbar=True,\n",
    "             sharex=False)    # 半径表示相应点的人口规模(对应着s)，用颜色表示价格(采用jet颜色表表示)，\n",
    "plt.legend()   # 从中我们可以看出 价格跟位置和人口密度有很大的关系，和ocean_proximity同样有关系，因此，从直觉上，我们可以考虑使用聚类算法。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "housing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:04.405282Z",
     "start_time": "2023-02-07T03:44:04.400516Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# 计算每个属性与median_house_value的相关性如何\n",
    "housing_num=housing.drop(\"ocean_proximity\",axis=1)\n",
    "corr_matrix = housing_num.corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:04.409569Z",
     "start_time": "2023-02-07T03:44:04.406457Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "corr_matrix[\"median_house_value\"].sort_values(\n",
    "    ascending=False)   # 每个属性与median_house_value的相关性"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:05.870911Z",
     "start_time": "2023-02-07T03:44:04.410341Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "from pandas.plotting import scatter_matrix\n",
    "attributes = [\"median_house_value\", \"median_income\",\n",
    "              \"total_rooms\", \"housing_median_age\"]\n",
    "\n",
    "# 类似用笛卡尔积的形式比较俩俩属性,对角线是相同属性的比较(且是直方图形式),其他的都是散点图\n",
    "scatter_matrix(housing[attributes], figsize=(20, 12), marker='o')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:05.960990Z",
     "start_time": "2023-02-07T03:44:05.871813Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# 把median_income与median_house_value比较的图提取出来\n",
    "housing.plot(kind=\"scatter\", x=\"median_income\",\n",
    "             y=\"median_house_value\", alpha=0.1)\n",
    "# 在图中,这俩者具有很强的相关系,我们发现好像有好几条平行线,我们可能要将它删除,"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### 实验不同属性的组合\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:05.975817Z",
     "start_time": "2023-02-07T03:44:05.962124Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:05.979693Z",
     "start_time": "2023-02-07T03:44:05.976759Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "housing[\"rooms_per_household\"] = housing[\"total_rooms\"] / \\\n",
    "    housing[\"households\"]   # 每个家庭拥有的房间数\n",
    "housing[\"bedrooms_per_room\"] = housing[\"total_bedrooms\"] / \\\n",
    "    housing[\"total_rooms\"]  # 每个房间的卧室数\n",
    "housing[\"population_per_household\"] = housing[\"population\"] / \\\n",
    "    housing[\"households\"]   # 每个家庭的人口数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:05.989653Z",
     "start_time": "2023-02-07T03:44:05.981292Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# 进行相关性分析\n",
    "corr_matrix = housing_num.corr()\n",
    "corr_matrix[\"median_house_value\"].sort_values(\n",
    "    ascending=False)   # 与median_house_value进行相关性分析"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:05.997115Z",
     "start_time": "2023-02-07T03:44:05.990625Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "corr_matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## 数据清洗\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:06.001291Z",
     "start_time": "2023-02-07T03:44:05.998670Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# 只会创建一个缺少median_house_value属性的副本\n",
    "housing = strat_train_set.drop(\"median_house_value\", axis=1)\n",
    "housing_labels = strat_train_set[\"median_house_value\"].copy()  # 存储刚刚删除属性的标签"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:06.011244Z",
     "start_time": "2023-02-07T03:44:06.002819Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "housing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-05-02T14:13:05.029485Z",
     "start_time": "2021-05-02T14:13:04.990590Z"
    },
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### 空值处理\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:06.017391Z",
     "start_time": "2023-02-07T03:44:06.013599Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# 一般空值处理有三种方法\n",
    "# 1: 放弃这些相应的区域\n",
    "# 2: 放弃整个属性\n",
    "# 3: 将缺少的值用0或平均数或中位数填充\n",
    "# # 1\n",
    "# housing.dropna(subset=[\"total_bedrooms\"])   删除相应的部分\n",
    "# # 2\n",
    "# housing.drop(\"total_bedrooms\",axis=1)    将整个属性删除\n",
    "# 3\n",
    "median = housing[\"total_bedrooms\"].median()   # 获取中位数\n",
    "housing[\"total_bedrooms\"].fillna(median, inplace=True)   # 将空值进行填充"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:06.385850Z",
     "start_time": "2023-02-07T03:44:06.018252Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# 使用scikit-learn处理缺失值\n",
    "from sklearn.impute import SimpleImputer\n",
    "\n",
    "imputer = SimpleImputer(strategy=\"median\")  # 它的数据必须是数值的,不能有文本属性\n",
    "\n",
    "# 中位数只能在数值属性上计算,所以我们需要创建一个没有ocean_proximity文本属性的数据副本\n",
    "housing_num = housing.drop(\"ocean_proximity\", axis=1)  # 删除ocean_proximity属性\n",
    "imputer.fit(housing_num)   # 计算得到的值存储在statistic里面"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:06.389942Z",
     "start_time": "2023-02-07T03:44:06.386819Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# 查看一下\n",
    "imputer.statistics_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:06.396409Z",
     "start_time": "2023-02-07T03:44:06.391498Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# 对比\n",
    "housing_num.median().values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:06.401094Z",
     "start_time": "2023-02-07T03:44:06.397381Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# 开始将缺失值进行填充\n",
    "X = imputer.transform(housing_num)  # 填充缺失值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:06.403844Z",
     "start_time": "2023-02-07T03:44:06.402030Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# 将其转换为DataFrame类型\n",
    "housing_tr = pd.DataFrame(X, columns=housing_num.columns,\n",
    "                          index=housing_num.index)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### 处理文本与分类属性\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:06.414066Z",
     "start_time": "2023-02-07T03:44:06.404618Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "housing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:06.416858Z",
     "start_time": "2023-02-07T03:44:06.415309Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# housing_cat=housing[\"ocean_proximity\"]\n",
    "# print(type(housing_cat))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:06.419404Z",
     "start_time": "2023-02-07T03:44:06.417892Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# housing_cat=housing[[\"ocean_proximity\"]]\n",
    "# print(type(housing_cat))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:06.427298Z",
     "start_time": "2023-02-07T03:44:06.420432Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import OrdinalEncoder\n",
    "\n",
    "housing_cat = housing[[\"ocean_proximity\"]]  # 得到的是DataFrame类型数据\n",
    "print(housing_cat)\n",
    "ordinal_encoder = OrdinalEncoder()\n",
    "housing_cat_encoded = ordinal_encoder.fit_transform(housing_cat)\n",
    "print(ordinal_encoder.categories_)  # 打印需要转换的文本信息属性\n",
    "housing_cat_encoded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:06.433994Z",
     "start_time": "2023-02-07T03:44:06.428630Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# 上面的OrdinalEncoder类也可以解决\n",
    "# 一般都是数值型的数据,将文本转换数值型的数据\n",
    "# 对于不是数值的属性值，sk页提供了转换方法使用sklearn中LabelEncoder类\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "encoder = LabelEncoder()   # 创建一个对象\n",
    "housing_cat = housing[\"ocean_proximity\"]  # 得到的是Series类型\n",
    "housing_cat_encoded = encoder.fit_transform(housing_cat)\n",
    "print(housing_cat_encoded)   # 转换后的编号\n",
    "print(encoder.classes_)   # 按照类型编号\n",
    "housing_cat_encoded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:06.439116Z",
     "start_time": "2023-02-07T03:44:06.435276Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# 在上边的例子中有个很大的问题，ml的算法会任务0和1比较接近，但是<1H OCEAN和NEAR OCEAN更相似\n",
    "# 为了解决这个问题，需要引入one hot的方式，用所在的位置设为1,其他的位置为0\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "\n",
    "encoder = OneHotEncoder()\n",
    "housing_cat_1hot = encoder.fit_transform(housing_cat_encoded.reshape(-1, 1))\n",
    "print(type(housing_cat_1hot))   # 属于一种Scipy稀疏矩阵\n",
    "print(housing_cat_1hot.toarray())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:06.467290Z",
     "start_time": "2023-02-07T03:44:06.440076Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# 也可以把label和one hot的步骤合成一个\n",
    "from sklearn.preprocessing import LabelBinarizer\n",
    "\n",
    "encoder = LabelBinarizer()\n",
    "housing_cat_1hot = encoder.fit_transform(housing_cat)\n",
    "print(housing_cat_1hot)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:06.476848Z",
     "start_time": "2023-02-07T03:44:06.468274Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "housing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### 自定义转换器\n",
    "\n",
    "- 我们也可以自定义一种 Transform,让它像 scikit 里面的 transform 一样好用\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:06.484252Z",
     "start_time": "2023-02-07T03:44:06.479761Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "housing.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:06.489881Z",
     "start_time": "2023-02-07T03:44:06.485289Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "housing.values[:, 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:06.499431Z",
     "start_time": "2023-02-07T03:44:06.490868Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# 自定义Transformation\n",
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "\n",
    "rooms_ix, bedrooms_ix, population_ix, household_ix = 3, 4, 5, 6  # 选择的列数\n",
    "\n",
    "\n",
    "# 这个自定义的transform是基于BaseEstimator,TransformerMixin\n",
    "class CombinedAttributesAdder(BaseEstimator, TransformerMixin):\n",
    "    # add_bedrooms_per_room这个参数让你更好的知道这个属性是否有助于机器学习\n",
    "    def __init__(self, add_bedrooms_per_room=True):\n",
    "        self.add_bedrooms_per_room = add_bedrooms_per_room\n",
    "\n",
    "    def fit(self, X, y=None):  # fit方法\n",
    "        return self\n",
    "\n",
    "    def transform(self, X):   # transform方法\n",
    "        print(\"==============\")\n",
    "        rooms_per_household = X[:, rooms_ix] / \\\n",
    "            X[:, household_ix]    # 每个家庭拥有的房间数\n",
    "        population_per_household = X[:, population_ix] / \\\n",
    "            X[:, household_ix]  # 每个家庭有几个人\n",
    "        if self.add_bedrooms_per_room:\n",
    "            bedrooms_per_room = X[:, bedrooms_ix] / \\\n",
    "                X[:, rooms_ix]   # 每个房间内拥有几个卧室\n",
    "            print(\"aaaa\", np.c_[X, rooms_per_household,\n",
    "                  population_per_household, bedrooms_per_room][0])\n",
    "            # 在同水平的方向进行连接\n",
    "            return np.c_[X, rooms_per_household, population_per_household, bedrooms_per_room]\n",
    "        else:\n",
    "            # 在同水平的方向进行连接\n",
    "            return np.c_[X, rooms_per_household, population_per_household]\n",
    "\n",
    "\n",
    "attr_adder = CombinedAttributesAdder()\n",
    "housing_extra_attribs = attr_adder.transform(housing.values)\n",
    "print(len(housing_extra_attribs[0]))  # 在每一行的后边拼接了两个值\n",
    "print(housing_extra_attribs)  # 在每一行的后边拼接了两个值"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### 特征缩放\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "- 对于机器学习，数据的 scaling 同样很重要，不同 scaling 的特征，会产生不同的结果，在我们的数据中，就存在 scaling 不一致的问题，解决这样的问题一般有两种方式:\n",
    "  - Min-max scaling，也叫 normalization， 主要是把值压缩到 0~1 之间，用值减去最小值后，再除以最大值减最小值的值\n",
    "  - Standardization，减去均值后再除以方差，这个跟也叫 normalization 不一样的地方在于，他的取值范围不是 0~1，它可以避免数据中存在极大值造成的误差\n",
    "- sklearn 提供了 StandardScaler 模块用于特征缩放，我们使用的是第二种 Standardization。\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### 转换流水线\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:06.518156Z",
     "start_time": "2023-02-07T03:44:06.500392Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "# 类似一个转换流水线器\n",
    "num_pipeline = Pipeline([\n",
    "    ('imputer', SimpleImputer(strategy=\"median\")),    # 用中位数填充空值\n",
    "    ('attribs_adder', CombinedAttributesAdder()),   # 添加组合后的数据\n",
    "    (('std_scaler', StandardScaler()))    # 特征缩放\n",
    "])\n",
    "housing_num_tr = num_pipeline.fit_transform(housing_num)    # 类似按步骤来做了上面三个操作\n",
    "housing_num_tr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:06.541525Z",
     "start_time": "2023-02-07T03:44:06.519238Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.compose import ColumnTransformer\n",
    "\n",
    "num_attribs = list(housing_num)\n",
    "print(num_attribs)\n",
    "cat_attribs = [\"ocean_proximity\"]\n",
    "\n",
    "full_pipeline = ColumnTransformer([\n",
    "    # 添加组合属性      名字(不要含_符号)  转换器  转换器能够应用的列名字或索引的列表\n",
    "    (\"num\", num_pipeline, num_attribs),\n",
    "    (\"cat\", OneHotEncoder(), cat_attribs),   # 文本转换为数字\n",
    "])\n",
    "housing_prepared = full_pipeline.fit_transform(housing)\n",
    "housing_prepared"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:06.633175Z",
     "start_time": "2023-02-07T03:44:06.542608Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# 我们上边的一系列过程，包含数据清洗，属性重组，数据缩放，text类型的转换，\n",
    "# 都可以使用sklearn的Pipeline来组合成一个整体的过程，支持异步的方式，同时进行多个pipeline\n",
    "\n",
    "# 使用属性组合的方式\n",
    "from sklearn.pipeline import FeatureUnion\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "\n",
    "class DataFrameSelector(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, attribute_names):\n",
    "        self.attribute_names = attribute_names\n",
    "\n",
    "    def fit(self, X, y=None):\n",
    "        return self\n",
    "\n",
    "    def transform(self, X):\n",
    "        return X[self.attribute_names].values\n",
    "\n",
    "\n",
    "class CustomLabelBinarizer(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, *args, **kwargs):\n",
    "        self.encoder = LabelBinarizer(*args, **kwargs)\n",
    "\n",
    "    def fit(self, x, y=None):\n",
    "        self.encoder.fit(x)\n",
    "        return self\n",
    "\n",
    "    def transform(self, x, y=None):\n",
    "        print(self.encoder.transform(x))\n",
    "        return self.encoder.transform(x)\n",
    "\n",
    "\n",
    "num_attribs = list(housing_num)\n",
    "print(num_attribs)\n",
    "cat_attribs = [\"ocean_proximity\"]\n",
    "\n",
    "\n",
    "num_pipeline = Pipeline([(\"selector\", DataFrameSelector(num_attribs)),    # 筛选num_attribs里面的特征\n",
    "                         (\"imputer\", SimpleImputer(strategy=\"median\")),   # 用中位数填充空值\n",
    "                         (\"attribs_adder\", CombinedAttributesAdder()),   # 添加组合的数据\n",
    "                         (\"std_scaler\", StandardScaler())])    # 特征缩放\n",
    "\n",
    "cat_pipeline = Pipeline([(\"selector\", DataFrameSelector(cat_attribs)),  # 筛选文本特征\n",
    "                        (\"label_binarizer\", CustomLabelBinarizer())])  # 进行文本转换,并使用onehot\n",
    "print(cat_pipeline.fit_transform(housing))\n",
    "# FeatureUnion 将多个转换器组合成一个新的对象\n",
    "full_pipeline = FeatureUnion(transformer_list=[(\"num_pipeline\", num_pipeline),\n",
    "                                               (\"cat_pipeline\", cat_pipeline)])\n",
    "\n",
    "housing_prepared = full_pipeline.fit_transform(housing)\n",
    "print(\"===================================================\")\n",
    "print(housing_prepared)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## 选择与训练模型\n",
    "\n",
    "- 做一个线性回归模型\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:06.636941Z",
     "start_time": "2023-02-07T03:44:06.633965Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "housing_prepared.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:06.640430Z",
     "start_time": "2023-02-07T03:44:06.638060Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "housing_prepared   # 经过数据清洗等操作之后的数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:06.644365Z",
     "start_time": "2023-02-07T03:44:06.641325Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "housing_labels  # median_house_value的数据"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### 训练和评估训练集\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:06.656342Z",
     "start_time": "2023-02-07T03:44:06.645352Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# 我们先用线性回归模型试一下\n",
    "from sklearn.linear_model import LinearRegression\n",
    "\n",
    "\n",
    "lin_reg = LinearRegression()\n",
    "lin_reg.fit(housing_prepared, housing_labels)   # 开始训练一个线性回归模型\n",
    "\n",
    "# 准备一些测试数据\n",
    "some_data = housing.iloc[:5]\n",
    "some_labels = housing_labels.iloc[:5]\n",
    "some_data_prepared = full_pipeline.transform(some_data)\n",
    "print(some_data_prepared)\n",
    "print(\"Predictions:\\t\", lin_reg.predict(some_data_prepared))    # 线性回归模型的预测结果\n",
    "print(\"Labels:\\t\\t,\", list(some_labels))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "从上面可知,预测结果跟真实结果还是有较大差别的,下面我们来采用代价函数 RMSE 进行评估\n",
    "\\\n",
    "![RMSE](images/代价函数.png)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:06.661578Z",
     "start_time": "2023-02-07T03:44:06.657815Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import mean_squared_error    # 均方根误差\n",
    "\n",
    "housing_predictions = lin_reg.predict(housing_prepared)  # 使用测试集进行线性回归预测\n",
    "lin_mse = mean_squared_error(\n",
    "    housing_labels, housing_predictions)   # 真实值与预测值进行计算MSE\n",
    "lin_rmse = np.sqrt(lin_mse)   # 开平方根计算RMSE\n",
    "lin_rmse  # 这种错误误差已经很大，说明当前的features不能提供预测的足够的信息或者当前模型不够强大"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "由上图可知,误差有点太大了,我们需要修正欠拟合,通常采用三种方法,第一种选择更强大的模型,第二种为算法训练提供更好的特征,第三种减少对模型的限制等方法\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:06.945274Z",
     "start_time": "2023-02-07T03:44:06.662470Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# 我们考虑第一种方法,采用决策树模型进行训练\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "\n",
    "tree_reg = DecisionTreeRegressor()\n",
    "tree_reg.fit(housing_prepared, housing_labels)   # 开始训练决策树模型\n",
    "\n",
    "tree_predictions = tree_reg.predict(housing_prepared)\n",
    "tree_mse = mean_squared_error(housing_labels, tree_predictions)   # 计算MSE\n",
    "tree_rmse = np.sqrt(tree_mse)    # 计算RMSE\n",
    "tree_rmse   # 当误差为0时表示可能出现了过拟合的情况了 (此时应该考虑使用交叉验证的方法)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### 使用交叉验证进行评估\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "对决策树进行评估,判断是否过拟合了,一般考虑使用交叉验证方法\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:08.377356Z",
     "start_time": "2023-02-07T03:44:06.946267Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# 上边出现了error为0的情况，说明过拟合了，可以使用sklearn的交叉验证\n",
    "# 把训练数据分成一定的分数，相互验证\n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "scores = cross_val_score(tree_reg, housing_prepared, housing_labels,\n",
    "                         scoring=\"neg_mean_squared_error\", cv=10)    # 进行10-折交叉验证\n",
    "tree_rmse_scores = np.sqrt(-scores)   # 得到的MSE是一个负数\n",
    "\n",
    "\n",
    "def display_scores(scores):\n",
    "    print(\"Scores:\", scores)\n",
    "    print(\"Mean:\", scores.mean())\n",
    "    print(\"Standard deviation:\", scores.std())\n",
    "\n",
    "\n",
    "display_scores(tree_rmse_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:08.447317Z",
     "start_time": "2023-02-07T03:44:08.378232Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# 对线性回归模型进行评估\n",
    "# 使用交叉验证看看回归的error\n",
    "line_scores = cross_val_score(lin_reg, housing_prepared, housing_labels,\n",
    "                              scoring=\"neg_mean_squared_error\", cv=10)\n",
    "line_rmse_scores = np.sqrt(-line_scores)\n",
    "\n",
    "\n",
    "display_scores(line_rmse_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:44:18.950781Z",
     "start_time": "2023-02-07T03:44:08.448257Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# 使用随机森林模型,并对其进行评估\n",
    "# 随机森林\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "random_forest = RandomForestRegressor()\n",
    "random_forest.fit(housing_prepared, housing_labels)\n",
    "\n",
    "forest_predictions = random_forest.predict(housing_prepared)\n",
    "forest_mse = mean_squared_error(housing_labels, forest_predictions)\n",
    "forest_rmse = np.sqrt(forest_mse)\n",
    "forest_rmse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:45:47.953510Z",
     "start_time": "2023-02-07T03:44:18.951615Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# 对随机森林模型进行评估\n",
    "# 使用交叉验证看看回归的error\n",
    "line_scores = cross_val_score(random_forest, housing_prepared, housing_labels,\n",
    "                              scoring=\"neg_mean_squared_error\", cv=10)\n",
    "line_rmse_scores = np.sqrt(-line_scores)\n",
    "\n",
    "\n",
    "display_scores(line_rmse_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:45:58.849262Z",
     "start_time": "2023-02-07T03:45:47.954337Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.svm import SVR\n",
    "\n",
    "svm_reg = SVR(kernel=\"linear\")    # 创建一个SVM模型\n",
    "svm_reg.fit(housing_prepared, housing_labels)   # 开始训练svm模型\n",
    "housing_predictions = svm_reg.predict(housing_prepared)   # 预测\n",
    "svm_mse = mean_squared_error(housing_labels, housing_predictions)   # 计算mse\n",
    "svm_rmse = np.sqrt(svm_mse)   # 计算rmse\n",
    "svm_rmse"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### 存储尝试过的模型\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:45:59.690472Z",
     "start_time": "2023-02-07T03:45:58.850098Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "! pip install joblib"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## 微调模型\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### 网格搜索\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:46:30.025272Z",
     "start_time": "2023-02-07T03:45:59.692659Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# 在得到一系列可用的模型列表后，需要对该模型做微调\n",
    "# Grid Search 网络搜索，使用sk对各种不同的参数组合做训练，获取最佳参数组合\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "# 网格搜索类似一种穷举法,在你给的参数里面一个月一个试,并且采用了5-折交叉验证\n",
    "param_grid = [{'n_estimators': [3, 10, 30], 'max_features': [2, 4, 6, 8]},\n",
    "              {'bootstrap': [False], 'n_estimators': [3, 10], 'max_features': [2, 3, 4]}]\n",
    "forest_reg = RandomForestRegressor()\n",
    "grid_search = GridSearchCV(forest_reg, param_grid,\n",
    "                           cv=5, scoring='neg_mean_squared_error')\n",
    "grid_search.fit(housing_prepared, housing_labels)\n",
    "\n",
    "grid_search.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:46:30.039552Z",
     "start_time": "2023-02-07T03:46:30.026109Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "grid_search_cv_results = pd.DataFrame(grid_search.cv_results_)\n",
    "grid_search_cv_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:46:30.054227Z",
     "start_time": "2023-02-07T03:46:30.041110Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# 获取最优的estimator(估算器)\n",
    "print(\"=========================\")\n",
    "print(grid_search.best_estimator_)\n",
    "print(\"=========================\")\n",
    "cvres = grid_search.cv_results_\n",
    "print(\"===================================================\")\n",
    "print(cvres[\"mean_test_score\"], cvres[\"params\"])\n",
    "print(\"===================================================\")\n",
    "for mean_score, params in zip(cvres[\"mean_test_score\"], cvres[\"params\"]):\n",
    "    print(np.sqrt(-mean_score), params)    # 打印每次训练评估分数和对应的参数"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### 随机搜索\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:48:08.992290Z",
     "start_time": "2023-02-07T03:46:30.055700Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# 随机搜索其实跟网格搜索差不多,只不过可能次数没有网格搜索那么多,因为它是随机的\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from scipy.stats import randint\n",
    "\n",
    "param_distribs = {\n",
    "    'n_estimators': randint(low=1, high=200),\n",
    "    'max_features': randint(low=1, high=8),\n",
    "}\n",
    "\n",
    "# random_state=42是随机状态,它可以是每次生成随机森林模型时都是不一样的\n",
    "forest_reg = RandomForestRegressor()\n",
    "rnd_search = RandomizedSearchCV(forest_reg, param_distributions=param_distribs,\n",
    "                                n_iter=10, cv=5, scoring='neg_mean_squared_error')\n",
    "rnd_search.fit(housing_prepared, housing_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:48:08.995351Z",
     "start_time": "2023-02-07T03:48:08.993172Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "rnd_search.best_params_   # 查看最优的参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:48:09.005061Z",
     "start_time": "2023-02-07T03:48:08.996248Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "rnd_search_cv_results = pd.DataFrame(rnd_search.cv_results_)\n",
    "rnd_search_cv_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:48:09.009742Z",
     "start_time": "2023-02-07T03:48:09.006160Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# 查看每个参数的评估分数\n",
    "cvres = rnd_search.cv_results_\n",
    "for mean_score, params in zip(cvres[\"mean_test_score\"], cvres[\"params\"]):\n",
    "    print(np.sqrt(-mean_score), params)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### 集成方法\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:48:09.014598Z",
     "start_time": "2023-02-07T03:48:09.011776Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# 在第七章将会详细的讲解"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### 分析最佳模型与误差\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:48:09.023549Z",
     "start_time": "2023-02-07T03:48:09.015785Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# 展示每个属性的相对重要程度\n",
    "feature_importances = grid_search.best_estimator_.feature_importances_\n",
    "feature_importances   # 展示每个属性的重要程度"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:48:09.029544Z",
     "start_time": "2023-02-07T03:48:09.025995Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "housing_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:48:09.046987Z",
     "start_time": "2023-02-07T03:48:09.042391Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "housing_prepared"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:48:09.060175Z",
     "start_time": "2023-02-07T03:48:09.048385Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "housing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:48:09.066561Z",
     "start_time": "2023-02-07T03:48:09.062488Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# 把上面的重要程度添加到属性的旁边\n",
    "# extra_attribs = [\"rooms_per_hhold\", \"pop_per_hhold\", \"bedrooms_per_room\"]\n",
    "# # cat_encoder = cat_pipeline.named_steps[\"cat_encoder\"] # old solution\n",
    "# cat_encoder = full_pipeline.named_transformers_[\"cat\"]\n",
    "# cat_one_hot_attribs = list(cat_encoder.categories_[0])\n",
    "# attributes = num_attribs + extra_attribs + cat_one_hot_attribs\n",
    "# sorted(zip(feature_importances, attributes), reverse=True)\n",
    "full_pipeline.verbose"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### 通过测试集评估系统\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:48:09.104712Z",
     "start_time": "2023-02-07T03:48:09.068354Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# 开始用测试集进行测试\n",
    "final_model = grid_search.best_estimator_\n",
    "\n",
    "X_test = strat_test_set.drop(\"median_house_value\", axis=1)\n",
    "y_test = strat_test_set[\"median_house_value\"].copy()\n",
    "\n",
    "X_test_prepared = full_pipeline.transform(X_test)\n",
    "final_predictions = final_model.predict(X_test_prepared)\n",
    "\n",
    "final_mse = mean_squared_error(y_test, final_predictions)   # 计算mse\n",
    "final_rmse = np.sqrt(final_mse)  # 计算rmse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:48:09.108731Z",
     "start_time": "2023-02-07T03:48:09.106070Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "final_rmse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:48:09.114985Z",
     "start_time": "2023-02-07T03:48:09.109831Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# 使用scipy.stats.t.interval()计算泛化误差的95%置信区间\n",
    "from scipy import stats\n",
    "\n",
    "confidence = 0.95\n",
    "squared_errors = (final_predictions - y_test) ** 2\n",
    "np.sqrt(stats.t.interval(confidence, len(squared_errors) - 1,\n",
    "                         loc=squared_errors.mean(),\n",
    "                         scale=stats.sem(squared_errors)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:48:09.119280Z",
     "start_time": "2023-02-07T03:48:09.116018Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "m = len(squared_errors)\n",
    "mean = squared_errors.mean()\n",
    "tscore = stats.t.ppf((1 + confidence) / 2, df=m - 1)\n",
    "tmargin = tscore * squared_errors.std(ddof=1) / np.sqrt(m)\n",
    "np.sqrt(mean - tmargin), np.sqrt(mean + tmargin)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-02-07T03:48:09.123471Z",
     "start_time": "2023-02-07T03:48:09.120255Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "zscore = stats.norm.ppf((1 + confidence) / 2)\n",
    "zmargin = zscore * squared_errors.std(ddof=1) / np.sqrt(m)\n",
    "np.sqrt(mean - zmargin), np.sqrt(mean + zmargin)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## 启动,监控和维护我的系统\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## 试试看\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## 练习题\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.18"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {
    "height": "91px",
    "width": "173px"
   },
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "303.531px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
